{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Comparison of Categorical Variable Encoding\n",
    "\n",
    "In this lecture, I will compare the performance of the different feature transformations we learnt to replace categorical variables.\n",
    "\n",
    "I will evaluate:\n",
    "\n",
    "- One hot encoding\n",
    "- Replacing labels by the count\n",
    "- Ordering labels according to target\n",
    "- Replacing labels by Risk\n",
    "- WoE\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import re # to extract text from string\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "from sklearn.metrics import roc_auc_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Name</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Ticket</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Braund, Mr. Owen Harris</td>\n",
       "      <td>male</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>A/5 21171</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Cumings, Mrs. John Bradley (Florence Briggs Th...</td>\n",
       "      <td>female</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>PC 17599</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>C85</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>Heikkinen, Miss. Laina</td>\n",
       "      <td>female</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>STON/O2. 3101282</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Futrelle, Mrs. Jacques Heath (Lily May Peel)</td>\n",
       "      <td>female</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>113803</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>C123</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Allen, Mr. William Henry</td>\n",
       "      <td>male</td>\n",
       "      <td>35.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>373450</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PassengerId  Survived  Pclass  \\\n",
       "0            1         0       3   \n",
       "1            2         1       1   \n",
       "2            3         1       3   \n",
       "3            4         1       1   \n",
       "4            5         0       3   \n",
       "\n",
       "                                                Name     Sex   Age  SibSp  \\\n",
       "0                            Braund, Mr. Owen Harris    male  22.0      1   \n",
       "1  Cumings, Mrs. John Bradley (Florence Briggs Th...  female  38.0      1   \n",
       "2                             Heikkinen, Miss. Laina  female  26.0      0   \n",
       "3       Futrelle, Mrs. Jacques Heath (Lily May Peel)  female  35.0      1   \n",
       "4                           Allen, Mr. William Henry    male  35.0      0   \n",
       "\n",
       "   Parch            Ticket     Fare Cabin Embarked  \n",
       "0      0         A/5 21171   7.2500   NaN        S  \n",
       "1      0          PC 17599  71.2833   C85        C  \n",
       "2      0  STON/O2. 3101282   7.9250   NaN        S  \n",
       "3      0            113803  53.1000  C123        S  \n",
       "4      0            373450   8.0500   NaN        S  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# let's load again the titanic dataset\n",
    "\n",
    "data = pd.read_csv('titanic.csv')\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PassengerId      int64\n",
       "Survived         int64\n",
       "Pclass           int64\n",
       "Name            object\n",
       "Sex             object\n",
       "Age            float64\n",
       "SibSp            int64\n",
       "Parch            int64\n",
       "Ticket          object\n",
       "Fare           float64\n",
       "Cabin           object\n",
       "Embarked        object\n",
       "dtype: object"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# let's check how many numerical and categorical variables we have\n",
    "\n",
    "data.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# let's drop non-important columns:\n",
    "# name and ticket have too many different categories, and Passenger Id is a unique key, non informative\n",
    "\n",
    "data.drop(['PassengerId', 'Name', 'Ticket'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>male</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>female</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>C85</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>female</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>female</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>C123</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>male</td>\n",
       "      <td>35.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Survived  Pclass     Sex   Age  SibSp  Parch     Fare Cabin Embarked\n",
       "0         0       3    male  22.0      1      0   7.2500   NaN        S\n",
       "1         1       1  female  38.0      1      0  71.2833   C85        C\n",
       "2         1       3  female  26.0      0      0   7.9250   NaN        S\n",
       "3         1       1  female  35.0      1      0  53.1000  C123        S\n",
       "4         0       3    male  35.0      0      0   8.0500   NaN        S"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# let's inspect the dataset\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Survived      0\n",
       "Pclass        0\n",
       "Sex           0\n",
       "Age         177\n",
       "SibSp         0\n",
       "Parch         0\n",
       "Fare          0\n",
       "Cabin       687\n",
       "Embarked      2\n",
       "dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# let's check for missing data\n",
    "\n",
    "data.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Embarked and Cabin have missing date and are categorical\n",
    "# let's fill NA values of categorical variables with an additional label\n",
    "\n",
    "data.Cabin.fillna('Missing', inplace=True)\n",
    "data.Embarked.fillna('Missing', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>male</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>M</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>female</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>C</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>female</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>M</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>female</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>C</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>male</td>\n",
       "      <td>35.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>M</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Survived  Pclass     Sex   Age  SibSp  Parch     Fare Cabin Embarked\n",
       "0         0       3    male  22.0      1      0   7.2500     M        S\n",
       "1         1       1  female  38.0      1      0  71.2833     C        C\n",
       "2         1       3  female  26.0      0      0   7.9250     M        S\n",
       "3         1       1  female  35.0      1      0  53.1000     C        S\n",
       "4         0       3    male  35.0      0      0   8.0500     M        S"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Now we extract the first letter of the cabin\n",
    "\n",
    "data['Cabin'] = data['Cabin'].astype(str).str[0]\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((623, 9), (268, 9))"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Let's divide into train and test set\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(data, data.Survived,\n",
    "                                                    test_size=0.3,\n",
    "                                                    random_state=0)\n",
    "X_train.shape, X_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Sole\\Anaconda3\\lib\\site-packages\\pandas\\core\\generic.py:3549: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  self._update_inplace(new_data)\n"
     ]
    }
   ],
   "source": [
    "# Let's replace null values in age by a value at right end of distribution\n",
    "\n",
    "def impute_na(df, variable, extreme):\n",
    "    df[variable].fillna(extreme, inplace=True)\n",
    "    \n",
    "impute_na(X_train, 'Age', X_train.Age.mean()+3*X_train.Age.std())\n",
    "impute_na(X_test, 'Age', X_train.Age.mean()+3*X_train.Age.std()) # note how we calculate the end of the distribution on train set "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>857</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>51.00000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>26.5500</td>\n",
       "      <td>E</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>female</td>\n",
       "      <td>49.00000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>76.7292</td>\n",
       "      <td>D</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>386</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>male</td>\n",
       "      <td>1.00000</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>46.9000</td>\n",
       "      <td>M</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>124</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>54.00000</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>77.2875</td>\n",
       "      <td>D</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>578</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>female</td>\n",
       "      <td>73.43632</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>14.4583</td>\n",
       "      <td>M</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Survived  Pclass     Sex       Age  SibSp  Parch     Fare Cabin Embarked\n",
       "857         1       1    male  51.00000      0      0  26.5500     E        S\n",
       "52          1       1  female  49.00000      1      0  76.7292     D        C\n",
       "386         0       3    male   1.00000      5      2  46.9000     M        S\n",
       "124         0       1    male  54.00000      0      1  77.2875     D        S\n",
       "578         0       3  female  73.43632      1      0  14.4583     M        C"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Survived    0\n",
       " Pclass      0\n",
       " Sex         0\n",
       " Age         0\n",
       " SibSp       0\n",
       " Parch       0\n",
       " Fare        0\n",
       " Cabin       0\n",
       " Embarked    0\n",
       " dtype: int64, Survived    0\n",
       " Pclass      0\n",
       " Sex         0\n",
       " Age         0\n",
       " SibSp       0\n",
       " Parch       0\n",
       " Fare        0\n",
       " Cabin       0\n",
       " Embarked    0\n",
       " dtype: int64)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# let's check that we have no missing data after NA imputation\n",
    "\n",
    "X_train.isnull().sum(), X_test.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### One Hot Encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Sex_male</th>\n",
       "      <th>Cabin_B</th>\n",
       "      <th>Cabin_C</th>\n",
       "      <th>Cabin_D</th>\n",
       "      <th>Cabin_E</th>\n",
       "      <th>Cabin_F</th>\n",
       "      <th>Cabin_G</th>\n",
       "      <th>Cabin_M</th>\n",
       "      <th>Cabin_T</th>\n",
       "      <th>Embarked_Missing</th>\n",
       "      <th>Embarked_Q</th>\n",
       "      <th>Embarked_S</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>857</th>\n",
       "      <td>51.00000</td>\n",
       "      <td>26.5500</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>49.00000</td>\n",
       "      <td>76.7292</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>386</th>\n",
       "      <td>1.00000</td>\n",
       "      <td>46.9000</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>124</th>\n",
       "      <td>54.00000</td>\n",
       "      <td>77.2875</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>578</th>\n",
       "      <td>73.43632</td>\n",
       "      <td>14.4583</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          Age     Fare  Pclass  Sex_male  Cabin_B  Cabin_C  Cabin_D  Cabin_E  \\\n",
       "857  51.00000  26.5500       1         1        0        0        0        1   \n",
       "52   49.00000  76.7292       1         0        0        0        1        0   \n",
       "386   1.00000  46.9000       3         1        0        0        0        0   \n",
       "124  54.00000  77.2875       1         1        0        0        1        0   \n",
       "578  73.43632  14.4583       3         0        0        0        0        0   \n",
       "\n",
       "     Cabin_F  Cabin_G  Cabin_M  Cabin_T  Embarked_Missing  Embarked_Q  \\\n",
       "857        0        0        0        0                 0           0   \n",
       "52         0        0        0        0                 0           0   \n",
       "386        0        0        1        0                 0           0   \n",
       "124        0        0        0        0                 0           0   \n",
       "578        0        0        1        0                 0           0   \n",
       "\n",
       "     Embarked_S  \n",
       "857           1  \n",
       "52            0  \n",
       "386           1  \n",
       "124           1  \n",
       "578           0  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_OHE(df):\n",
    "    df_OHE = pd.concat([df[['Age', 'Fare', 'Pclass']], \n",
    "                         pd.get_dummies(df[[ 'Sex', 'Cabin', 'Embarked']], drop_first=True)],\n",
    "                        axis=1\n",
    "                       )\n",
    "    return df_OHE\n",
    "\n",
    "X_train_OHE = get_OHE(X_train)\n",
    "X_test_OHE = get_OHE(X_test)\n",
    "\n",
    "X_train_OHE.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Sex_male</th>\n",
       "      <th>Cabin_B</th>\n",
       "      <th>Cabin_C</th>\n",
       "      <th>Cabin_D</th>\n",
       "      <th>Cabin_E</th>\n",
       "      <th>Cabin_F</th>\n",
       "      <th>Cabin_G</th>\n",
       "      <th>Cabin_M</th>\n",
       "      <th>Embarked_Q</th>\n",
       "      <th>Embarked_S</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>495</th>\n",
       "      <td>103.157714</td>\n",
       "      <td>14.4583</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>648</th>\n",
       "      <td>103.157714</td>\n",
       "      <td>7.5500</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>278</th>\n",
       "      <td>7.000000</td>\n",
       "      <td>29.1250</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>103.157714</td>\n",
       "      <td>146.5208</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>255</th>\n",
       "      <td>29.000000</td>\n",
       "      <td>15.2458</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            Age      Fare  Pclass  Sex_male  Cabin_B  Cabin_C  Cabin_D  \\\n",
       "495  103.157714   14.4583       3         1        0        0        0   \n",
       "648  103.157714    7.5500       3         1        0        0        0   \n",
       "278    7.000000   29.1250       3         1        0        0        0   \n",
       "31   103.157714  146.5208       1         0        1        0        0   \n",
       "255   29.000000   15.2458       3         0        0        0        0   \n",
       "\n",
       "     Cabin_E  Cabin_F  Cabin_G  Cabin_M  Embarked_Q  Embarked_S  \n",
       "495        0        0        0        1           0           0  \n",
       "648        0        0        0        1           0           1  \n",
       "278        0        0        0        1           1           0  \n",
       "31         0        0        0        0           0           0  \n",
       "255        0        0        0        1           0           0  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test_OHE.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can see that the test set does not contain the variables Cabin_T and Embarked_Missing. This occurred because those labels are indeed rare, and therefore, when splitting train and test set, they were only left on the train set. \n",
    "\n",
    "Perhaps, it would be a good idea to replace them by the most frequent label, as we learnt in previous sections."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((623, 9), (268, 9))"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# for now, I will just generate the missing column in the test set and fill it with 0s\n",
    "# given that no observation in the test set is positive for those labels\n",
    "\n",
    "X_test_OHE.loc[:, 'Cabin_T'] = 0\n",
    "X_test_OHE.loc[:, 'Embarked_Missing'] = 0\n",
    "\n",
    "X_train.shape, X_test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Categorical to counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>857</th>\n",
       "      <td>1</td>\n",
       "      <td>408</td>\n",
       "      <td>51.00000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>26.5500</td>\n",
       "      <td>27</td>\n",
       "      <td>457</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>1</td>\n",
       "      <td>215</td>\n",
       "      <td>49.00000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>76.7292</td>\n",
       "      <td>26</td>\n",
       "      <td>108</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>386</th>\n",
       "      <td>3</td>\n",
       "      <td>408</td>\n",
       "      <td>1.00000</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>46.9000</td>\n",
       "      <td>471</td>\n",
       "      <td>457</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>124</th>\n",
       "      <td>1</td>\n",
       "      <td>408</td>\n",
       "      <td>54.00000</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>77.2875</td>\n",
       "      <td>26</td>\n",
       "      <td>457</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>578</th>\n",
       "      <td>3</td>\n",
       "      <td>215</td>\n",
       "      <td>73.43632</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>14.4583</td>\n",
       "      <td>471</td>\n",
       "      <td>108</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Pclass  Sex       Age  SibSp  Parch     Fare  Cabin  Embarked\n",
       "857       1  408  51.00000      0      0  26.5500     27       457\n",
       "52        1  215  49.00000      1      0  76.7292     26       108\n",
       "386       3  408   1.00000      5      2  46.9000    471       457\n",
       "124       1  408  54.00000      0      1  77.2875     26       457\n",
       "578       3  215  73.43632      1      0  14.4583    471       108"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def categorical_to_counts(df_train, df_test):\n",
    "    # make a temporary copy of the original dataframes\n",
    "    df_train_temp = df_train.copy()\n",
    "    df_test_temp = df_test.copy()\n",
    "    \n",
    "    for col in ['Sex', 'Cabin', 'Embarked']:\n",
    "        # make the dictionary mapping label to counts\n",
    "        X_frequency_map = df_train_temp[col].value_counts().to_dict()\n",
    "        \n",
    "        # remap the labels to their counts\n",
    "        df_train_temp[col] = df_train_temp[col].map(X_frequency_map)\n",
    "        df_test_temp[col] = df_test_temp[col].map(X_frequency_map)\n",
    "    \n",
    "    # drop the target\n",
    "    df_train_temp.drop(['Survived'], axis=1, inplace=True)\n",
    "    df_test_temp.drop(['Survived'], axis=1, inplace=True)\n",
    "    \n",
    "    return df_train_temp, df_test_temp\n",
    "\n",
    "X_train_count, X_test_count = categorical_to_counts(X_train, X_test)\n",
    "X_train_count.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Categorical to ordered numbers according to target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>857</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>51.00000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>26.5500</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>49.00000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>76.7292</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>386</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1.00000</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>46.9000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>124</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>54.00000</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>77.2875</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>578</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>73.43632</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>14.4583</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Pclass  Sex       Age  SibSp  Parch     Fare  Cabin  Embarked\n",
       "857       1    0  51.00000      0      0  26.5500      7         0\n",
       "52        1    1  49.00000      1      0  76.7292      6         2\n",
       "386       3    0   1.00000      5      2  46.9000      1         0\n",
       "124       1    0  54.00000      0      1  77.2875      6         0\n",
       "578       3    1  73.43632      1      0  14.4583      1         2"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def categories_to_ordered(df_train, df_test):\n",
    "    # make a temporary copy of the datasets \n",
    "    df_train_temp = df_train.copy()\n",
    "    df_test_temp = df_test.copy()\n",
    "    \n",
    "    for col in [ 'Sex', 'Cabin', 'Embarked']:\n",
    "        # order the labels according to target mean\n",
    "        ordered_labels = df_train.groupby([col])['Survived'].mean().sort_values().index\n",
    "        \n",
    "        # create the dictionary to map the ordered labels to an ordinal number\n",
    "        ordinal_label = {k:i for i, k in enumerate(ordered_labels, 0)} \n",
    "        \n",
    "        # remap the categories  to these ordinal numbers\n",
    "        df_train_temp[col] = df_train[col].map(ordinal_label)\n",
    "        df_test_temp[col] = df_test[col].map(ordinal_label)\n",
    "    \n",
    "    # remove the target\n",
    "    df_train_temp.drop(['Survived'], axis=1, inplace=True)\n",
    "    df_test_temp.drop(['Survived'], axis=1, inplace=True)\n",
    "    \n",
    "    return df_train_temp, df_test_temp\n",
    "        \n",
    "X_train_ordered, X_test_ordered = categories_to_ordered(X_train, X_test)\n",
    "X_train_ordered.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Categorical to risk factor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>857</th>\n",
       "      <td>1</td>\n",
       "      <td>0.196078</td>\n",
       "      <td>51.00000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>26.5500</td>\n",
       "      <td>0.740741</td>\n",
       "      <td>0.341357</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>1</td>\n",
       "      <td>0.753488</td>\n",
       "      <td>49.00000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>76.7292</td>\n",
       "      <td>0.692308</td>\n",
       "      <td>0.564815</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>386</th>\n",
       "      <td>3</td>\n",
       "      <td>0.196078</td>\n",
       "      <td>1.00000</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>46.9000</td>\n",
       "      <td>0.303609</td>\n",
       "      <td>0.341357</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>124</th>\n",
       "      <td>1</td>\n",
       "      <td>0.196078</td>\n",
       "      <td>54.00000</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>77.2875</td>\n",
       "      <td>0.692308</td>\n",
       "      <td>0.341357</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>578</th>\n",
       "      <td>3</td>\n",
       "      <td>0.753488</td>\n",
       "      <td>73.43632</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>14.4583</td>\n",
       "      <td>0.303609</td>\n",
       "      <td>0.564815</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Pclass       Sex       Age  SibSp  Parch     Fare     Cabin  Embarked\n",
       "857       1  0.196078  51.00000      0      0  26.5500  0.740741  0.341357\n",
       "52        1  0.753488  49.00000      1      0  76.7292  0.692308  0.564815\n",
       "386       3  0.196078   1.00000      5      2  46.9000  0.303609  0.341357\n",
       "124       1  0.196078  54.00000      0      1  77.2875  0.692308  0.341357\n",
       "578       3  0.753488  73.43632      1      0  14.4583  0.303609  0.564815"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def categories_to_risk(df_train, df_test):\n",
    "    # temporary copy of the original dataframes\n",
    "    df_train_temp = df_train.copy()\n",
    "    df_test_temp = df_test.copy()\n",
    "    \n",
    "    for col in ['Sex', 'Cabin', 'Embarked']:\n",
    "        # make a dictionary mapping labels to the mean target for that label\n",
    "        risk_dict = df_train.groupby([col])['Survived'].mean().to_dict()\n",
    "        \n",
    "        # re-map the labels\n",
    "        df_train_temp[col] = df_train[col].map(risk_dict)\n",
    "        df_test_temp[col] = df_test[col].map(risk_dict)\n",
    "    \n",
    "    # drop the target\n",
    "    df_train_temp.drop(['Survived'], axis=1, inplace=True)\n",
    "    df_test_temp.drop(['Survived'], axis=1, inplace=True)        \n",
    "    return df_train_temp, df_test_temp\n",
    "        \n",
    "X_train_risk, X_test_risk = categories_to_risk(X_train, X_test)\n",
    "X_train_risk.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### WoE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>857</th>\n",
       "      <td>1</td>\n",
       "      <td>-1.410987</td>\n",
       "      <td>51.00000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>26.5500</td>\n",
       "      <td>1.049822</td>\n",
       "      <td>-0.657254</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>1</td>\n",
       "      <td>1.117304</td>\n",
       "      <td>49.00000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>76.7292</td>\n",
       "      <td>0.810930</td>\n",
       "      <td>0.260726</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>386</th>\n",
       "      <td>3</td>\n",
       "      <td>-1.410987</td>\n",
       "      <td>1.00000</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>46.9000</td>\n",
       "      <td>-0.830169</td>\n",
       "      <td>-0.657254</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>124</th>\n",
       "      <td>1</td>\n",
       "      <td>-1.410987</td>\n",
       "      <td>54.00000</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>77.2875</td>\n",
       "      <td>0.810930</td>\n",
       "      <td>-0.657254</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>578</th>\n",
       "      <td>3</td>\n",
       "      <td>1.117304</td>\n",
       "      <td>73.43632</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>14.4583</td>\n",
       "      <td>-0.830169</td>\n",
       "      <td>0.260726</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Pclass       Sex       Age  SibSp  Parch     Fare     Cabin  Embarked\n",
       "857       1 -1.410987  51.00000      0      0  26.5500  1.049822 -0.657254\n",
       "52        1  1.117304  49.00000      1      0  76.7292  0.810930  0.260726\n",
       "386       3 -1.410987   1.00000      5      2  46.9000 -0.830169 -0.657254\n",
       "124       1 -1.410987  54.00000      0      1  77.2875  0.810930 -0.657254\n",
       "578       3  1.117304  73.43632      1      0  14.4583 -0.830169  0.260726"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def categories_to_woe(df_train, df_test):\n",
    "    # temporary copy of datafranes\n",
    "    df_train_temp = df_train.copy()\n",
    "    df_test_temp = df_test.copy()\n",
    "    \n",
    "    for col in [ 'Sex', 'Cabin', 'Embarked']:\n",
    "        # create df containing the different parts of the WoE equation\n",
    "        prob_df = X_train.groupby([col])['Survived'].mean()\n",
    "        prob_df = pd.DataFrame(prob_df)\n",
    "        prob_df['Died'] = 1-prob_df.Survived\n",
    "        \n",
    "        # remove zero values as the log of zero does not exist\n",
    "        prob_df.loc[prob_df.Died == 0, 'Died'] = 0.00001\n",
    "        prob_df.loc[prob_df.Survived == 0, 'Survived'] = 0.00001\n",
    "        \n",
    "        # calculate WoE\n",
    "        prob_df['WoE'] = np.log(prob_df.Survived/prob_df.Died)\n",
    "        woe = prob_df['WoE'].to_dict()\n",
    "        \n",
    "        # re-map the labels to WoE\n",
    "        df_train_temp[col] = df_train[col].map(woe)\n",
    "        df_test_temp[col] = df_test[col].map(woe)\n",
    "    \n",
    "    # drop the target\n",
    "    df_train_temp.drop(['Survived'], axis=1, inplace=True)\n",
    "    df_test_temp.drop(['Survived'], axis=1, inplace=True)        \n",
    "    return df_train_temp, df_test_temp\n",
    "        \n",
    "X_train_woe, X_test_woe = categories_to_woe(X_train, X_test)\n",
    "X_train_woe.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Random Forest Performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# create a function to build random forests and compare performance in train and test set\n",
    "\n",
    "def run_randomForests(X_train, X_test, y_train, y_test):\n",
    "    rf = RandomForestClassifier(n_estimators=200, random_state=39, max_depth=4)\n",
    "    rf.fit(X_train, y_train)\n",
    "    print('Train set')\n",
    "    pred = rf.predict_proba(X_train)\n",
    "    print('Random Forests roc-auc: {}'.format(roc_auc_score(y_train, pred[:,1])))\n",
    "    print('Test set')\n",
    "    pred = rf.predict_proba(X_test)\n",
    "    print('Random Forests roc-auc: {}'.format(roc_auc_score(y_test, pred[:,1])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train set\n",
      "Random Forests roc-auc: 0.8823669768551659\n",
      "Test set\n",
      "Random Forests roc-auc: 0.8705059523809523\n"
     ]
    }
   ],
   "source": [
    "# OHE\n",
    "run_randomForests(X_train_OHE, X_test_OHE, y_train, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train set\n",
      "Random Forests roc-auc: 0.8935435240016485\n",
      "Test set\n",
      "Random Forests roc-auc: 0.8820833333333333\n"
     ]
    }
   ],
   "source": [
    "# counts\n",
    "run_randomForests(X_train_count, X_test_count, y_train, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train set\n",
      "Random Forests roc-auc: 0.8931530769397628\n",
      "Test set\n",
      "Random Forests roc-auc: 0.8794345238095238\n"
     ]
    }
   ],
   "source": [
    "# ordered labels\n",
    "run_randomForests(X_train_ordered, X_test_ordered, y_train, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train set\n",
      "Random Forests roc-auc: 0.8931422311880436\n",
      "Test set\n",
      "Random Forests roc-auc: 0.879375\n"
     ]
    }
   ],
   "source": [
    "# risk factor\n",
    "run_randomForests(X_train_risk, X_test_risk, y_train, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train set\n",
      "Random Forests roc-auc: 0.8931422311880436\n",
      "Test set\n",
      "Random Forests roc-auc: 0.879375\n"
     ]
    }
   ],
   "source": [
    "# woe\n",
    "run_randomForests(X_train_woe, X_test_woe, y_train, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Comparing the roc_auc values on the test sets, we can see that we obtain the best performance of random forests when we train the algorithms on the variables engineered with some intelligence: this is, replacing labels by risk, WoE or ordered numbers. Although, the performances of the different trees for the different categorical engineering methods are quite similar.\n",
    "\n",
    "### Logistic Regression Performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def run_logistic(X_train, X_test, y_train, y_test):\n",
    "    # function to train and test the performance of logistic regression\n",
    "    logit = LogisticRegression(random_state=44)\n",
    "    logit.fit(X_train, y_train)\n",
    "    print('Train set')\n",
    "    pred = logit.predict_proba(X_train)\n",
    "    print('Logistic Regression roc-auc: {}'.format(roc_auc_score(y_train, pred[:,1])))\n",
    "    print('Test set')\n",
    "    pred = logit.predict_proba(X_test)\n",
    "    print('Logistic Regression roc-auc: {}'.format(roc_auc_score(y_test, pred[:,1])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train set\n",
      "Logistic Regression roc-auc: 0.8518524543936141\n",
      "Test set\n",
      "Logistic Regression roc-auc: 0.8429761904761905\n"
     ]
    }
   ],
   "source": [
    "# OHE\n",
    "run_logistic(X_train_OHE, X_test_OHE, y_train, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train set\n",
      "Logistic Regression roc-auc: 0.8546343897095507\n",
      "Test set\n",
      "Logistic Regression roc-auc: 0.8510119047619048\n"
     ]
    }
   ],
   "source": [
    "# counts\n",
    "run_logistic(X_train_count, X_test_count, y_train, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train set\n",
      "Logistic Regression roc-auc: 0.8561419491984991\n",
      "Test set\n",
      "Logistic Regression roc-auc: 0.8505357142857143\n"
     ]
    }
   ],
   "source": [
    "# ordered labels\n",
    "run_logistic(X_train_ordered, X_test_ordered, y_train, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train set\n",
      "Logistic Regression roc-auc: 0.8531919047309168\n",
      "Test set\n",
      "Logistic Regression roc-auc: 0.8498214285714285\n"
     ]
    }
   ],
   "source": [
    "# risk factor\n",
    "run_logistic(X_train_risk, X_test_risk, y_train, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train set\n",
      "Logistic Regression roc-auc: 0.8555996616125464\n",
      "Test set\n",
      "Logistic Regression roc-auc: 0.8514285714285714\n"
     ]
    }
   ],
   "source": [
    "# woe\n",
    "run_logistic(X_train_woe, X_test_woe, y_train, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For Logistic regression, the best performance is obtained when we replace the categories by their counts.\n",
    "\n",
    "### Gradient Boosting Trees Performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "\n",
    "def run_xgb(X_train, X_test, y_train, y_test):\n",
    "    # function to train and test gradient boosting trees\n",
    "    # using the xgboost package\n",
    "    \n",
    "    xgb_model = XGBClassifier(random_state=44, max_depth=2)\n",
    "    xgb_model.fit(X_train, y_train)\n",
    "    print('Train set')\n",
    "    pred = xgb_model.predict_proba(X_train)\n",
    "    print('Logistic Regression roc-auc: {}'.format(roc_auc_score(y_train, pred[:,1])))\n",
    "    print('Test set')\n",
    "    pred = xgb_model.predict_proba(X_test)\n",
    "    print('Logistic Regression roc-auc: {}'.format(roc_auc_score(y_test, pred[:,1])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train set\n",
      "Logistic Regression roc-auc: 0.9004414220949655\n",
      "Test set\n",
      "Logistic Regression roc-auc: 0.8898214285714285\n"
     ]
    }
   ],
   "source": [
    "# OHE\n",
    "\n",
    "# the xgb package requires that both datasets show the columns in the same order, so I pass\n",
    "# the columns in brackets to make sure this is the case\n",
    "\n",
    "run_xgb(X_train_OHE[X_train_OHE.columns], X_test_OHE[X_train_OHE.columns], y_train, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train set\n",
      "Logistic Regression roc-auc: 0.9082503633326826\n",
      "Test set\n",
      "Logistic Regression roc-auc: 0.885297619047619\n"
     ]
    }
   ],
   "source": [
    "# counts\n",
    "run_xgb(X_train_count, X_test_count, y_train, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train set\n",
      "Logistic Regression roc-auc: 0.9063198195266913\n",
      "Test set\n",
      "Logistic Regression roc-auc: 0.8844642857142858\n"
     ]
    }
   ],
   "source": [
    "# ordered labels\n",
    "run_xgb(X_train_ordered, X_test_ordered, y_train, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train set\n",
      "Logistic Regression roc-auc: 0.9063198195266913\n",
      "Test set\n",
      "Logistic Regression roc-auc: 0.8844642857142858\n"
     ]
    }
   ],
   "source": [
    "# risk factor\n",
    "run_xgb(X_train_risk, X_test_risk, y_train, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train set\n",
      "Logistic Regression roc-auc: 0.9063198195266913\n",
      "Test set\n",
      "Logistic Regression roc-auc: 0.8844642857142858\n"
     ]
    }
   ],
   "source": [
    "# woe\n",
    "run_xgb(X_train_woe, X_test_woe, y_train, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For xgboost, we obtain the highest performance when the features are encoded using OHE.\n",
    "\n",
    "### Conclusions\n",
    "\n",
    "Different machine learning algorithms benefit of different ways of preprocessing categorical variables. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prepare a submission for Kaggle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Name</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Ticket</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "      <th>Survived</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>892</td>\n",
       "      <td>3</td>\n",
       "      <td>Kelly, Mr. James</td>\n",
       "      <td>male</td>\n",
       "      <td>34.5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>330911</td>\n",
       "      <td>7.8292</td>\n",
       "      <td>Missing</td>\n",
       "      <td>Q</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>893</td>\n",
       "      <td>3</td>\n",
       "      <td>Wilkes, Mrs. James (Ellen Needs)</td>\n",
       "      <td>female</td>\n",
       "      <td>47.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>363272</td>\n",
       "      <td>7.0000</td>\n",
       "      <td>Missing</td>\n",
       "      <td>S</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>894</td>\n",
       "      <td>2</td>\n",
       "      <td>Myles, Mr. Thomas Francis</td>\n",
       "      <td>male</td>\n",
       "      <td>62.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>240276</td>\n",
       "      <td>9.6875</td>\n",
       "      <td>Missing</td>\n",
       "      <td>Q</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>895</td>\n",
       "      <td>3</td>\n",
       "      <td>Wirz, Mr. Albert</td>\n",
       "      <td>male</td>\n",
       "      <td>27.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>315154</td>\n",
       "      <td>8.6625</td>\n",
       "      <td>Missing</td>\n",
       "      <td>S</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>896</td>\n",
       "      <td>3</td>\n",
       "      <td>Hirvonen, Mrs. Alexander (Helga E Lindqvist)</td>\n",
       "      <td>female</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3101298</td>\n",
       "      <td>12.2875</td>\n",
       "      <td>Missing</td>\n",
       "      <td>S</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PassengerId  Pclass                                          Name     Sex  \\\n",
       "0          892       3                              Kelly, Mr. James    male   \n",
       "1          893       3              Wilkes, Mrs. James (Ellen Needs)  female   \n",
       "2          894       2                     Myles, Mr. Thomas Francis    male   \n",
       "3          895       3                              Wirz, Mr. Albert    male   \n",
       "4          896       3  Hirvonen, Mrs. Alexander (Helga E Lindqvist)  female   \n",
       "\n",
       "    Age  SibSp  Parch   Ticket     Fare    Cabin Embarked  Survived  \n",
       "0  34.5      0      0   330911   7.8292  Missing        Q         0  \n",
       "1  47.0      1      0   363272   7.0000  Missing        S         0  \n",
       "2  62.0      0      0   240276   9.6875  Missing        Q         0  \n",
       "3  27.0      0      0   315154   8.6625  Missing        S         0  \n",
       "4  22.0      1      1  3101298  12.2875  Missing        S         0  "
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# load data for submission\n",
    "data = pd.read_csv('titanic_submission.csv')\n",
    "\n",
    "# fillna in categorical variables\n",
    "data.Cabin.fillna('Missing', inplace=True)\n",
    "data.Embarked.fillna('Missing', inplace=True)\n",
    "\n",
    "# fillna in numerical variables\n",
    "impute_na(data, 'Age', X_train.Age.mean()+3*X_train.Age.std())\n",
    "\n",
    "# mock variable so I can run my function\n",
    "data.loc[:,'Survived'] = 0 # this will not alter the result as we calculate the imputation on the training set\n",
    "X_train_ordered, data_ordered = categories_to_ordered(X_train, data)\n",
    "\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Pclass', 'Sex', 'Age', 'SibSp', 'Parch', 'Fare', 'Cabin', 'Embarked'], dtype='object')"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# this are the columns I will use to make the predictions\n",
    "X_train_ordered.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Survived</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>892</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>893</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>894</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>895</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>896</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PassengerId  Survived\n",
       "0          892         0\n",
       "1          893         0\n",
       "2          894         0\n",
       "3          895         0\n",
       "4          896         0"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# here I will make the predictions using the encoding into ordered labels\n",
    "# go ahead and try different methods, and algorithms, and why not, average the results before submitting\n",
    "\n",
    "xgb_model = XGBClassifier(random_state=44, max_depth=2)\n",
    "xgb_model.fit(X_train_ordered, y_train)\n",
    "pred = xgb_model.predict(data_ordered[['Pclass', 'Sex', 'Age', 'SibSp', 'Parch', 'Fare', 'Cabin', 'Embarked']])\n",
    "temp = pd.concat([data.PassengerId, pd.Series(pred)], axis=1)\n",
    "temp.columns = ['PassengerId','Survived']\n",
    "temp.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "temp.to_csv('Kaggle_submission.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "We are now ready to submit the dataset to Kaggle."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": "block",
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
